#!/usr/bin/python
# -*- coding:utf-8 -*-

__author__="nMask"
__Date__="20170224"
__Blog__="http://thief.one"

import Queue
import threading
import time
import sys
import os
import urllib
import re
import codecs

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.keys import Keys #导入模拟点击

reload(sys)
sys.setdefaultencoding("utf-8")

# dcap = dict(DesiredCapabilities.PHANTOMJS)  #设置userAgent
# dcap["phantomjs.page.settings.userAgent"] = ("Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36")

def down_cb(blocknum, blocksize, totalsize):
    '''回调函数
    @blocknum: 已经下载的数据块
    @blocksize: 数据块的大小
    @totalsize: 远程文件的大小
    '''
    # percent = 100.0 * blocknum * blocksize / totalsize
    # if percent > 100:
    #     percent = 100

    # print "download file %.2f%%"% percent
    pass

if __name__=="__main__":

	'''
	用法：
	1.实例化类
	2.运行open_phantomjs 开启phantomjs进程
	3.运行getbody函数，传入url
	4.运行close_phantomjs 关闭phantomjs进程
	'''
	# url_list = ['https://shop137309399.m.taobao.com/',
	# 			'https://shop58176825.m.taobao.com/',
	# 			'https://metersbonwe.m.tmall.com/',
	# 			'https://shop65268300.m.taobao.com/',
	# 			'https://shop58356243.m.taobao.com',
	# 			'https://shop35995663.m.taobao.com/']

	url_list = ['https://shop137309399.m.taobao.com/']

	# browser = webdriver.PhantomJS(executable_path='C:\Python27\Scripts\phantomjs.exe')
	browser = webdriver.PhantomJS()
	# browser = webdriver.Chrome()

	#page.implicitly_wait(60)        ##设置超时时间
	#page.set_page_load_timeout(60)  ##设置超时时间

	for url in url_list:
		print url

		end_pos = url.find(".m.taobao")
		if end_pos == -1:
			end_pos = url.find(".m.tmall")
			if end_pos == -1:
				print 'is not taotao or tmall url'
				continue

		try:
			browser.get(url)
			browser.sleep(10)
			browser.save_screenshot("open.png")

			tag_name = url[8:end_pos]

			if not os.path.exists(tag_name):
				os.makedirs(tag_name)

			html_file_name = tag_name + ".html"
			print 'file = ', html_file_name
			doc = open(html_file_name, "w+")
			doc.write(browser.page_source)
			doc.close()

			# 获取资源文件
			# tag_list = [('//link', 'href'), ('//script', 'src'), ('//img', 'src')]
			tag_list = [('//img', 'src')]
			for tag_pair in tag_list:
				all_links = browser.find_elements_by_xpath(tag_pair[0])
				#print 'link element size = ', len(all_links)
				for link in all_links:
					source_url = link.get_attribute(tag_pair[1])
					if len(source_url) == 0:
						#print 'tag = %s, attr = %s, source_url empty, not download' % (tag_pair[0], tag_pair[1])
						continue

					# 以http或者//开始
					if source_url.find('http') != 0:
						if source_url.find('//') != 0:
							#print 'tag = %s, attr = %s, not begwith http or //, source_url = %s, not download' % (tag_pair[0], tag_pair[1], source_url)
							continue

					# 如果中间有逗号隔开的是多个文件，不需要下载
					if source_url.find('.js,') != -1:
						#print 'tag = %s, attr = %s, source_url = %s, is mul file, not download' % (tag_pair[0], tag_pair[1], source_url)	
						continue

					pos = source_url.find('.js&')
					if pos != -1:
						source_url = source_url[0:pos+3]
						print 'new_source_url = %s' % source_url

					print 'tag = %s, attr = %s, source_url = %s' % (tag_pair[0], tag_pair[1], source_url)
					source_name = os.path.basename(source_url)
					#print 'source_name = %s' % source_name
					save_path = os.path.join(tag_name, source_name)
					if os.path.exists(save_path):
						#print 'had exits %s' % save_path
						continue

					#print 'save_path = ', save_path
					# download file
					urllib.urlretrieve(source_url, save_path, down_cb)

		except Exception as e:
			print "[error]", e

	browser.quit()